From f9945926ebe70e4c1bb122bdb03f6336036cb9cf Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Sun, 21 Feb 2016 21:02:20 -0800 Subject: [PATCH] Stop downloading all packages immediately Instead delay all downloads until just before they're needed. This should ensure that we don't download any more packages than we really need to. --- src/cargo/core/package.rs | 12 ++++--- src/cargo/core/registry.rs | 15 ++------- src/cargo/ops/cargo_compile.rs | 4 +-- src/cargo/ops/cargo_fetch.rs | 16 +++++----- tests/test_cargo_registry.rs | 58 ++++++++++++++++++++++++++-------- 5 files changed, 65 insertions(+), 40 deletions(-) diff --git a/src/cargo/core/package.rs b/src/cargo/core/package.rs index b0c0c2461..c3ec0dd61 100644 --- a/src/cargo/core/package.rs +++ b/src/cargo/core/package.rs @@ -9,7 +9,7 @@ use semver::Version; use core::{Dependency, Manifest, PackageId, SourceId, Target}; use core::{Summary, Metadata, SourceMap}; use ops; -use util::{CargoResult, Config, LazyCell, ChainError, internal}; +use util::{CargoResult, Config, LazyCell, ChainError, internal, human}; use rustc_serialize::{Encoder,Encodable}; /// Information about a package that is available somewhere in the file system. @@ -125,11 +125,11 @@ pub struct PackageSet<'cfg> { } impl<'cfg> PackageSet<'cfg> { - pub fn new(packages: Vec, + pub fn new(package_ids: &[PackageId], sources: SourceMap<'cfg>) -> PackageSet<'cfg> { PackageSet { - packages: packages.into_iter().map(|pkg| { - (pkg.package_id().clone(), LazyCell::new(Some(pkg))) + packages: package_ids.iter().map(|id| { + (id.clone(), LazyCell::new(None)) }).collect(), sources: RefCell::new(sources), } @@ -151,7 +151,9 @@ impl<'cfg> PackageSet<'cfg> { let source = try!(sources.get_mut(id.source_id()).chain_error(|| { internal(format!("couldn't find source for `{}`", id)) })); - let pkg = try!(source.download(id)); + let pkg = try!(source.download(id).chain_error(|| { + human("unable to get packages from source") + })); assert!(slot.fill(pkg).is_ok()); Ok(slot.borrow().unwrap()) } diff --git a/src/cargo/core/registry.rs b/src/cargo/core/registry.rs index bd44d7787..f744f4937 100644 --- a/src/cargo/core/registry.rs +++ b/src/cargo/core/registry.rs @@ -2,7 +2,7 @@ use std::collections::{HashSet, HashMap}; use core::{Source, SourceId, SourceMap, Summary, Dependency, PackageId, Package}; use core::PackageSet; -use util::{CargoResult, ChainError, Config, human, internal, profile}; +use util::{CargoResult, ChainError, Config, human, profile}; /// Source of information about a group of packages. /// @@ -85,18 +85,9 @@ impl<'cfg> PackageRegistry<'cfg> { } } - pub fn get(mut self, package_ids: &[PackageId]) - -> CargoResult> { + pub fn get(self, package_ids: &[PackageId]) -> PackageSet<'cfg> { trace!("getting packages; sources={}", self.sources.len()); - - let pkgs = try!(package_ids.iter().map(|id| { - let src = try!(self.sources.get_mut(id.source_id()).chain_error(|| { - internal(format!("failed to find a source listed for `{}`", id)) - })); - src.download(id) - }).collect::>>()); - - Ok(PackageSet::new(pkgs, self.sources)) + PackageSet::new(package_ids, self.sources) } fn ensure_loaded(&mut self, namespace: &SourceId, kind: Kind) -> CargoResult<()> { diff --git a/src/cargo/ops/cargo_compile.rs b/src/cargo/ops/cargo_compile.rs index 7d0f858ea..d81f2b0a1 100644 --- a/src/cargo/ops/cargo_compile.rs +++ b/src/cargo/ops/cargo_compile.rs @@ -133,8 +133,8 @@ pub fn resolve_dependencies<'a>(root_package: &Package, try!(ops::resolve_with_previous(&mut registry, root_package, method, Some(&resolve), None)); - let packages = try!(ops::get_resolved_packages(&resolved_with_overrides, - registry)); + let packages = ops::get_resolved_packages(&resolved_with_overrides, + registry); Ok((packages, resolved_with_overrides)) } diff --git a/src/cargo/ops/cargo_fetch.rs b/src/cargo/ops/cargo_fetch.rs index 35002bac1..0617a68bf 100644 --- a/src/cargo/ops/cargo_fetch.rs +++ b/src/cargo/ops/cargo_fetch.rs @@ -3,7 +3,7 @@ use std::path::Path; use core::registry::PackageRegistry; use core::{Package, PackageId, Resolve, PackageSet}; use ops; -use util::{CargoResult, Config, human, ChainError}; +use util::{CargoResult, Config}; /// Executes `cargo fetch`. pub fn fetch<'a>(manifest_path: &Path, @@ -12,16 +12,16 @@ pub fn fetch<'a>(manifest_path: &Path, let package = try!(Package::for_path(manifest_path, config)); let mut registry = PackageRegistry::new(config); let resolve = try!(ops::resolve_pkg(&mut registry, &package)); - get_resolved_packages(&resolve, registry).map(move |packages| { - (resolve, packages) - }) + let packages = get_resolved_packages(&resolve, registry); + for id in resolve.iter() { + try!(packages.get(id)); + } + Ok((resolve, packages)) } pub fn get_resolved_packages<'a>(resolve: &Resolve, registry: PackageRegistry<'a>) - -> CargoResult> { + -> PackageSet<'a> { let ids: Vec = resolve.iter().cloned().collect(); - registry.get(&ids).chain_error(|| { - human("unable to get packages from source") - }) + registry.get(&ids) } diff --git a/tests/test_cargo_registry.rs b/tests/test_cargo_registry.rs index 793235ede..bbc531978 100644 --- a/tests/test_cargo_registry.rs +++ b/tests/test_cargo_registry.rs @@ -567,6 +567,7 @@ test!(login_with_no_cargo_dir { }); test!(bad_license_file { + Package::new("foo", "1.0.0").publish(); let p = project("all") .file("Cargo.toml", r#" [project] @@ -957,21 +958,52 @@ test!(update_same_prefix_oh_my_how_was_this_a_bug { execs().with_status(0)); }); -test!(use_semver { - let p = project("foo") - .file("Cargo.toml", r#" - [project] - name = "bar" - version = "0.5.0" - authors = [] +test!(use_semver { + let p = project("foo") + .file("Cargo.toml", r#" + [project] + name = "bar" + version = "0.5.0" + authors = [] - [dependencies] - foo = "1.2.3-alpha.0" - "#) - .file("src/main.rs", "fn main() {}"); - p.build(); + [dependencies] + foo = "1.2.3-alpha.0" + "#) + .file("src/main.rs", "fn main() {}"); + p.build(); - Package::new("foo", "1.2.3-alpha.0").publish(); + Package::new("foo", "1.2.3-alpha.0").publish(); assert_that(p.cargo("build"), execs().with_status(0)); }); + +test!(only_download_relevant { + let p = project("foo") + .file("Cargo.toml", r#" + [project] + name = "bar" + version = "0.5.0" + authors = [] + + [target.foo.dependencies] + foo = "*" + [dev-dependencies] + bar = "*" + [dependencies] + baz = "*" + "#) + .file("src/main.rs", "fn main() {}"); + p.build(); + + Package::new("foo", "0.1.0").publish(); + Package::new("bar", "0.1.0").publish(); + Package::new("baz", "0.1.0").publish(); + + assert_that(p.cargo("build"), + execs().with_status(0).with_stdout(&format!("\ +{updating} registry `[..]` +{downloading} baz v0.1.0 ([..]) +{compiling} baz v0.1.0 ([..]) +{compiling} bar v0.5.0 ([..]) +", downloading = DOWNLOADING, compiling = COMPILING, updating = UPDATING))); +}); -- 2.30.2